{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid character in identifier (<ipython-input-2-ace6d2158950>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-2-ace6d2158950>\"\u001b[1;36m, line \u001b[1;32m1\u001b[0m\n\u001b[1;33m    关于本次征途杯瑕疵检测算法以及模型的部分说明：\u001b[0m\n\u001b[1;37m                          ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid character in identifier\n"
     ]
    }
   ],
   "source": [
    "关于本次征途杯瑕疵检测算法以及模型的部分说明：\n",
    "队伍：NK-CV-Club\n",
    "\n",
    "编程思路：\n",
    "复赛中的任务是在非平坦区域中的瑕疵检测。具体而言，是在背景图片的颜色和形状较之初赛多样化复杂化的情况下，\n",
    "在其中标注出所有瑕疵所在的像素。考虑到背景非常复杂，使用无监督的训练方法可能无法取得非常良好的效果，\n",
    "因此，我们使用了有监督的图像分割的办法，通过考察瑕疵的形状以及颜色，使用程序在线地在好品图上生成瑕疵与对应的标签，\n",
    "并将其输入模型进行训练。考虑到复赛中的瑕疵大小各异，因此我们选用了擅长应对多尺度分割的res2net作为我们模型的主体，\n",
    "同时使用基于res2net的csf作为我们的模型。loss函数使用的是交叉熵函数。\n",
    "除此之外，在模型的瑕疵生成方面，为了让生成的瑕疵更加逼真，我们使用了高斯模糊和图片叠加的办法，\n",
    "让生成的瑕疵更像是真实的瑕疵。考虑到part1和part2的图片有较为严重的噪点现象，高斯模糊也可以针对这一点进行改善。\n",
    "\n",
    "核心代码之瑕疵生成：\n",
    "def get_random_shape():\n",
    "    out = []\n",
    "    black_type = random.randint(0, 9)\n",
    "    if black_type == 0:\n",
    "        src_tmp = np.array(np.zeros((128, 128)), dtype='uint8')\n",
    "        point_num = 10\n",
    "        point_list = []\n",
    "        left_x = random.randint(0, 127)\n",
    "        left_y = random.randint(0, 127)\n",
    "        for i in range(point_num):\n",
    "            center_x = random.randint(0, 5)\n",
    "            center_y = random.randint(0, 5)\n",
    "            point_list.append((left_x + center_x, left_y + center_y))\n",
    "        point_list = np.array(point_list)\n",
    "        hull = cv.convexHull(point_list)\n",
    "        # print(hull)\n",
    "        src_tmp = cv.drawContours(src_tmp, [hull], 0, 255, -1)\n",
    "        src_tmp = rotato(src_tmp, random.randint(0, 360), 1, hull[0][0])\n",
    "        out = src_tmp > 125\n",
    "        return np.where(out, 1.0, 0)\n",
    "    #中略\n",
    "    elif black_type == 3 or black_type == 6:\n",
    "        src_tmp = np.array(np.zeros((128, 128)), dtype='uint8')\n",
    "        point_num = 10\n",
    "        point_list = []\n",
    "        left_x = random.randint(0, 127)\n",
    "        left_y = random.randint(0, 127)\n",
    "        for i in range(point_num):\n",
    "            center_x = random.randint(0, 30)\n",
    "            center_y = random.randint(0, 1)\n",
    "            point_list.append((left_x + center_x, left_y + center_y))\n",
    "        point_list = np.array(point_list)\n",
    "        hull = cv.convexHull(point_list)\n",
    "        # print(hull)\n",
    "        src_tmp = cv.drawContours(src_tmp, [hull], 0, 255, -1)\n",
    "        src_tmp = rotato(src_tmp, random.randint(0, 360), 1, hull[0][0])\n",
    "        out = src_tmp > 125\n",
    "        return np.where(out, 1.0, 0)\n",
    "    elif black_type == 4:\n",
    "        src_tmp = np.array(np.zeros((128, 128)), dtype='uint8')\n",
    "        left_x = random.randint(0, 50)\n",
    "        left_y = random.randint(0, 50)\n",
    "        right_x = random.randint(80, 127)\n",
    "        right_y = random.randint(80, 127)\n",
    "        src_tmp = cv.line(src_tmp, (left_x, left_y), (right_x, right_y), 255, random.randint(1, 2), lineType=cv.LINE_AA)\n",
    "        src_tmp = rotato(src_tmp, random.randint(0, 360), 1, (64, 64))\n",
    "        out = src_tmp > 125\n",
    "        return np.where(out, 1.0, 0)\n",
    "    elif black_type == 5 or black_type == 7:\n",
    "        n = random.randint(3, 5)\n",
    "        r = 0.7\n",
    "        N = n * 3 + 1  # number of points in the Path\n",
    "        # There is the initial point and 3 points per cubic bezier curve. Thus, the curve will only pass though n points, which will be the sharp edges, the other 2 modify the shape of the bezier curve\n",
    "\n",
    "        angles = np.linspace(0, 2 * np.pi, N)\n",
    "        codes = np.full(N, Path.CURVE4)\n",
    "        codes[0] = Path.MOVETO\n",
    "\n",
    "        verts = np.stack((np.cos(angles), np.sin(angles))).T * (2 * r * np.random.random(N) + 1 - r)[:, None]\n",
    "        verts[-1, :] = verts[0, :]  # Using this instad of Path.CLOSEPOLY avoids an innecessary straight line\n",
    "        path = Path(verts, codes)\n",
    "\n",
    "        fig = plt.figure()\n",
    "        ax = fig.add_subplot(111)\n",
    "        patch = patches.PathPatch(path, facecolor='none', lw=2)\n",
    "        ax.add_patch(patch)\n",
    "\n",
    "        ax.set_xlim(np.min(verts) * 1.1, np.max(verts) * 1.1)\n",
    "        ax.set_ylim(np.min(verts) * 1.1, np.max(verts) * 1.1)\n",
    "        ax.axis('off')  # removes the axis to leave only the shape\n",
    "        plt.savefig('./1.png')\n",
    "        plt.close()\n",
    "        img = cv.imread('1.png', 0)\n",
    "        try:\n",
    "            _, c, h = cv.findContours(img, cv.RETR_TREE, cv.CHAIN_APPROX_NONE)\n",
    "        except:\n",
    "            c, h = cv.findContours(img, cv.RETR_TREE, cv.CHAIN_APPROX_NONE)\n",
    "        temp = np.zeros(img.shape, np.uint8) * 255\n",
    "\n",
    "        cv.drawContours(temp, c, 1, 255, -1)\n",
    "        temp = cv.resize(temp, (128, 128))\n",
    "        temp = np.where(temp > 128, get_weight((64, 64)), 0.)\n",
    "        src_tmp = rotato(temp, random.randint(0, 360), random.randint(4, 8) / 10,\n",
    "                         center=(random.randint(30, 100), random.randint(30, 100)))\n",
    "        return src_tmp\n",
    "\n",
    "核心代码之Bottle2neck：\n",
    "class Bottle2neck(nn.Module):\n",
    "    expansion = 4\n",
    "\n",
    "    def __init__(self, inplanes, planes, stride=1, dilation_=1, downsample=None, baseWidth=26, scale=4, stype='normal'):\n",
    "        \"\"\" Constructor\n",
    "        Args:\n",
    "            inplanes: input channel dimensionality\n",
    "            planes: output channel dimensionality\n",
    "            stride: conv stride. Replaces pooling layer.\n",
    "            downsample: None when stride = 1\n",
    "            baseWidth: basic width of conv3x3\n",
    "            scale: number of scale.\n",
    "            type: 'normal': normal set. 'stage': first block of a new stage.\n",
    "        \"\"\"\n",
    "        super(Bottle2neck, self).__init__()\n",
    "\n",
    "        width = int(math.floor(planes * (baseWidth / 64.0)))\n",
    "        self.conv1 = nn.Conv2d(inplanes, width * scale, kernel_size=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(width * scale, affine=affine_par)\n",
    "        for i in self.bn1.parameters():\n",
    "            i.requires_grad = False\n",
    "        if scale == 1:\n",
    "            self.nums = 1\n",
    "        else:\n",
    "            self.nums = scale - 1\n",
    "        if stype == 'stage':\n",
    "            self.pool = nn.AvgPool2d(kernel_size=3, stride=stride, padding=1)\n",
    "        convs = []\n",
    "        bns = []\n",
    "        for i in range(self.nums):\n",
    "            convs.append(nn.Conv2d(width, width, kernel_size=3, stride=stride, dilation=dilation_, padding=dilation_,\n",
    "                                   bias=False))\n",
    "            bns.append(nn.BatchNorm2d(width))\n",
    "        self.convs = nn.ModuleList(convs)\n",
    "        self.bns = nn.ModuleList(bns)\n",
    "        for j in range(self.nums):\n",
    "            for i in self.bns[j].parameters():\n",
    "                i.requires_grad = False\n",
    "\n",
    "        self.conv3 = nn.Conv2d(width * scale, planes * self.expansion, kernel_size=1, bias=False)\n",
    "        self.bn3 = nn.BatchNorm2d(planes * self.expansion)\n",
    "        for i in self.bn3.parameters():\n",
    "            i.requires_grad = False\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.downsample = downsample\n",
    "        self.stype = stype\n",
    "        self.scale = scale\n",
    "        self.width = width\n",
    "\n",
    "    def forward(self, x):\n",
    "        residual = x\n",
    "\n",
    "        out = self.conv1(x)\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "\n",
    "        spx = torch.split(out, self.width, 1)\n",
    "        for i in range(self.nums):\n",
    "            if i == 0 or self.stype == 'stage':\n",
    "                sp = spx[i]\n",
    "            else:\n",
    "                sp = sp + spx[i]\n",
    "            sp = self.convs[i](sp)\n",
    "            sp = self.relu(self.bns[i](sp))\n",
    "            if i == 0:\n",
    "                out = sp\n",
    "            else:\n",
    "                out = torch.cat((out, sp), 1)\n",
    "        if self.scale != 1 and self.stype == 'normal':\n",
    "            out = torch.cat((out, spx[self.nums]), 1)\n",
    "        elif self.scale != 1 and self.stype == 'stage':\n",
    "            out = torch.cat((out, self.pool(spx[self.nums])), 1)\n",
    "\n",
    "        out = self.conv3(out)\n",
    "        out = self.bn3(out)\n",
    "\n",
    "        if self.downsample is not None:\n",
    "            residual = self.downsample(x)\n",
    "\n",
    "        out += residual\n",
    "        out = self.relu(out)\n",
    "\n",
    "        return out\n",
    "\n",
    "核心代码之csfres2net：\n",
    "class Res2Net(nn.Module):\n",
    "    def __init__(self, block, layers, baseWidth=26, scale=4, input_channle=3, need_pretrain=False):\n",
    "        self.inplanes = 64\n",
    "        super(Res2Net, self).__init__()\n",
    "        self.baseWidth = baseWidth\n",
    "        self.scale = scale\n",
    "        self.conv1 = nn.Sequential(\n",
    "            nn.Conv2d(input_channle, 32, 3, 2, 1, bias=False),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(32, 32, 3, 1, 1, bias=False),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(32, 64, 3, 1, 1, bias=False)\n",
    "        )\n",
    "        self.bn1 = nn.BatchNorm2d(64, affine=affine_par)\n",
    "        for i in self.bn1.parameters():\n",
    "            i.requires_grad = False\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
    "        self.layer1 = self._make_layer(block, 64, layers[0])\n",
    "        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n",
    "        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n",
    "        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n",
    "        self.avgpool = nn.AvgPool2d(7, stride=1)\n",
    "        if need_pretrain:\n",
    "            self.fc = nn.Linear(512 * block.expansion, 1000)\n",
    "\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n",
    "            elif isinstance(m, nn.BatchNorm2d):\n",
    "                nn.init.constant_(m.weight, 1)\n",
    "                nn.init.constant_(m.bias, 0)\n",
    "        \n",
    "    #forward略\n",
    "\n",
    "class MSBlock(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, dilations=[1, 2, 4, 8, 16]):\n",
    "        super(MSBlock, self).__init__()\n",
    "        self.dilations = dilations\n",
    "        each_out_channels = out_channels // 5\n",
    "        self.msconv = nn.ModuleList()\n",
    "        for i in range(len(dilations)):\n",
    "            if i != len(dilations) - 1:\n",
    "                this_outc = each_out_channels\n",
    "            else:\n",
    "                this_outc = out_channels - each_out_channels * (len(dilations) - 1)\n",
    "            self.msconv.append(\n",
    "                nn.Conv2d(in_channels, this_outc, 3, padding=dilations[i], dilation=dilations[i], bias=False))\n",
    "        self.bn = nn.GroupNorm(32, out_channels)\n",
    "        self.prelu = nn.PReLU(out_channels)\n",
    "\n",
    "    def forward(self, x):\n",
    "        outs = []\n",
    "        for i in range(len(self.dilations)):\n",
    "            outs.append(self.msconv[i](x))\n",
    "        out = torch.cat(outs, dim=1)\n",
    "        del outs\n",
    "        out = self.prelu(self.bn(out))\n",
    "        return out\n",
    "\n",
    "\n",
    "class CSFNet(nn.Module):\n",
    "    def __init__(self, num_classes=1, input_channle=3, need_pretrain=None):\n",
    "        super(CSFNet, self).__init__()\n",
    "        self.base = Res2Net(Bottle2neck, [3, 4, 6, 3], baseWidth=26, scale=4, input_channle=input_channle,\n",
    "                            need_pretrain=(need_pretrain is not None))\n",
    "        if need_pretrain is not None:\n",
    "            self.base.load_state_dict(torch.load(need_pretrain))\n",
    "\n",
    "        fuse_in_channel = 256 + 512 + 1024 + 2048\n",
    "        fuse_in_split = [1 / 15, 2 / 15, 4 / 15, 8 / 15]\n",
    "        fuse_out_channel = 128 + 256 + 512 + 512\n",
    "        fuse_out_split = [1 / 11, 2 / 11, 4 / 11, 4 / 11]\n",
    "\n",
    "        self.fuse = gOctaveCBR(fuse_in_channel, fuse_out_channel, kernel_size=(1, 1), padding=0,\n",
    "                               alpha_in=fuse_in_split, alpha_out=fuse_out_split, stride=1)\n",
    "        self.ms = PallMSBlock(fuse_out_channel, fuse_out_channel, alpha=fuse_out_split)\n",
    "        self.fuse1x1 = gOctaveCBR(fuse_out_channel, fuse_out_channel, kernel_size=(1, 1), padding=0,\n",
    "                                  alpha_in=fuse_out_split, alpha_out=[1, ], stride=1)\n",
    "        self.cls_layer = nn.Conv2d(fuse_out_channel, num_classes, kernel_size=1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        features = self.base(x)\n",
    "        fuse = self.fuse(features)\n",
    "        fuse = self.ms(fuse)\n",
    "        fuse = self.fuse1x1(fuse)\n",
    "        output = self.cls_layer(fuse[0])\n",
    "        output = F.interpolate(output, x.size()[2:], mode='bilinear', align_corners=False)\n",
    "\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid character in identifier (<ipython-input-3-4025dc93eee8>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-3-4025dc93eee8>\"\u001b[1;36m, line \u001b[1;32m1\u001b[0m\n\u001b[1;33m    特色亮点：\u001b[0m\n\u001b[1;37m        ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid character in identifier\n"
     ]
    }
   ],
   "source": [
    "特色亮点：\n",
    "1，使用了监督学习方法，运用神经网络强大的拟合能力做到了端到端处理，超参数量较少。同时性能超越一般的基于图像生成的无监督学习方法。\n",
    "2，使用了在多尺度方面表现相当优秀的res2net作为基础。经过我们的实验，基于res2net的csf对比起同样是进行语义分割任务的FCN，deeplab模型来说，\n",
    "在漏检方面有更好的表现。换言之，res2net可以把面积稍大的瑕疵全部检出，而FCN和deeplab等模型只能检测出来部分，有相当严重的漏检情况。\n",
    "3，在生成瑕疵时，为了让生成的瑕疵更加接近真实的瑕疵，对于不规则的瑕疵，我们使用了贝塞尔曲线来生成不规则的瑕疵，在生成之后，以图形的中心为原点生成了高斯核作为瑕疵的权值，\n",
    "并以这个权值作为图片叠加的权值，实现了中间浓，边缘浅的瑕疵。对于规则的瑕疵，我们使用了随机描点，然后用findContours和drawContours染色的办法来实现。"
   ]
  },
  {
   "source": [
    "编程语言：\n",
    "\n",
    "Python==3.6 Pytorch==1.2.0\n",
    "\n",
    "软件运行环境：\n",
    "\n",
    "Linux python==3.6 pytorch==1.2.0 torchvision==0.7.0 opencv-python==4.1.2.30 opencv-contrib-python==3.4.2.17 pathLib2==2.3.5 matplotlib==3.1.1\n",
    "\n",
    "硬件配置环境：\n",
    "\n",
    "内存至少8G\n",
    "显卡：2080ti\n",
    "CPU： 9900kf\n",
    "整体训练+推测需要11小时\n",
    "在硬件环境改变的时候，运行时间随之改变\n",
    "\n",
    "运行说明：\n",
    "\n",
    "cd到code部分运行main.py即可在result中获得结果。\n",
    "运行细节如下：\n",
    "- main.py中首先调用瑕疵生成器gen1,gen2,gen3,这三个瑕疵生成器分别从OK大图中截取小图生成瑕疵并保存文件\n",
    "- 生成完成之后调用网络训练代码train_part1,train_part2,train_part3,这三个部分分别使用语义分割模块进行训练\n",
    "- 训练过程中会存储ckpt，只有调用p1,p2,p3,这3个模块进行端到端推测，得到初步的json结果\n",
    "- 最后一步，因为初步json结果可能存在过检或者瑕疵面积生成过小等问题，程序会调用filter代码进行导向滤波和形态学膨胀，最后输出最终json，压缩后放入result\n",
    "备注信息：\n",
    "\n",
    "在part3的模型的训练过程中，我们使用了预训练的模型为基础进行微调，以下是预训练模型的网址：\n",
    "https://shanghuagao.oss-cn-beijing.aliyuncs.com/res2net/res2net50_v1b_26w_4s-3cf99910.pth"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": null,
   "outputs": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}